{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "__author__ = 'Roger'\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "from matplotlib.pylab import rcParams\n",
    "from statsmodels.tsa.stattools import adfuller\n",
    "from statsmodels.graphics.tsaplots import plot_acf, plot_pacf\n",
    "from statsmodels.tsa.arima_model import ARIMA,ARMA\n",
    "import datetime\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "## draw arima predict result so that we can analysis easily\n",
    "## output the data\n",
    "\n",
    "# test for stationarity with adf\n",
    "def test_stationarity(timeseries):\n",
    "    # Determing rolling statistics\n",
    "#     rolmean = pd.rolling_mean(timeseries, window=window)\n",
    "#     rolstd = pd.rolling_std(timeseries, window=window)\n",
    "\n",
    "    # Plot rolling statistics\n",
    "#     orig = plt.plot(timeseries, color='blue',label='Original')\n",
    "#     mean = plt.plot(rolmean, color='red', label='Rolling Mean')\n",
    "#     std = plt.plot(rolstd, color='black', label = 'Rolling Std')\n",
    "#     plt.legend(loc='best')\n",
    "#     plt.title('Rolling Mean & Standard Deviation')\n",
    "#     plt.show(block=False)\n",
    "\n",
    "    # Perform Dickey-Fuller test:\n",
    "    print('Results of Dickey-Fuller Test:')\n",
    "    dftest = adfuller(timeseries, autolag='AIC')\n",
    "    dfoutput = pd.Series(dftest[0:4], index=['Test Statistic','p-value','#Lags Used','Number of Observations Used'])\n",
    "    for key,value in dftest[4].items():\n",
    "        dfoutput['Critical Value (%s)'%key] = value\n",
    "    print(dfoutput)\n",
    "    if dfoutput['Test Statistic'] < dfoutput['Critical Value (1%)']:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def load_data_as_npdict():\n",
    "    shop = {}\n",
    "    index = {}\n",
    "    begin_time = datetime.date(2015, 7, 1)\n",
    "    with open('../data/shopid_day_num.txt', 'r') as f:\n",
    "        for line in f.readlines():\n",
    "            item = line.strip().split(',')\n",
    "            shopId = int(item[0])\n",
    "            if shopId not in shop:\n",
    "                shop[shopId] = np.array(item[1:], 'f')\n",
    "\n",
    "    for i in range(489):\n",
    "        if begin_time + datetime.timedelta(i) != datetime.date(2015, 12, 12):\n",
    "            index[i] = begin_time + datetime.timedelta(i)\n",
    "    return index, shop\n",
    "\n",
    "index, shop = load_data_as_npdict()\n",
    "\n",
    "# test = [5,6,10,18,23,65]\n",
    "# test = [1,2,3,4,5,6]\n",
    "test = [18]\n",
    "\n",
    "# time series\n",
    "pd_index = pd.date_range('2016-9-1', periods=61)\n",
    "print(pd_index)\n",
    "\n",
    "# for i in test:\n",
    "#     df = pd.Series(shop[i][-61:],index = pd_index)\n",
    "#     pvalue = test_stationarity(df,7)\n",
    "#     print(pvalue)\n",
    "    # print(df.head())\n",
    "#     test_stationarity(df,8)\n",
    "#     draw_acf_pacf(df)\n",
    "\n",
    "for i in test:\n",
    "    df = pd.Series(shop[i][-61:],index = pd_index,dtype='f')\n",
    "    model = ARIMA(df,(0,0,3))\n",
    "    results_AR = model.fit(disp=-1, method='css')\n",
    "    result = results_AR.predict('2016-11-1', '2016-11-14', dynamic=True)\n",
    "    print(result.values)\n",
    "    result_s = pd.Series(result)\n",
    "    \n",
    "    df['2016-10-20'] = 1\n",
    "    df_log = np.log(df)\n",
    "    print(df)\n",
    "    plt.plot(df)\n",
    "    plt.plot(df_log, color='red')\n",
    "#     test_stationarity(df_log, 3)\n",
    "    \n",
    "    plt.plot(df)\n",
    "    plt.title('RSS: %.4f'% np.sum((results_AR.fittedvalues-df)**2))\n",
    "    plt.plot(results_AR.fittedvalues, color='red')\n",
    "    plt.plot(result, color='red')\n",
    "    plt.show()\n",
    "    \n",
    "for i in test:\n",
    "    df = pd.Series(shop[i][-61:],index = pd_index,dtype='f')\n",
    "#     model = ARIMA(df,(0,0,3))\n",
    "#     results_AR = model.fit(disp=-1, method='css')\n",
    "#     result = results_AR.predict('2016-11-1', '2016-11-8', dynamic=True)\n",
    "#     result_s = pd.Series(result)\n",
    "#     df.add(result_s)\n",
    "    \n",
    "    print(df)\n",
    "\n",
    "# diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# pd_index = pd.date_range('2016-10-1', periods=31)\n",
    "# df = pd.Series(shop[23][-31:],index = pd_index,dtype='f')\n",
    "# df_log = np.log(df)\n",
    "# test_stationarity(df,5)\n",
    "# test_stationarity(df_log,5)\n",
    "\n",
    "# difference\n",
    "def diff_ts(ts, d):\n",
    "    global shift_ts_list\n",
    "    #  dynamic predict the difference\n",
    "    global last_data_shift_list\n",
    "    shift_ts_list = []\n",
    "    last_data_shift_list = []\n",
    "    tmp_ts = ts\n",
    "    for i in d:\n",
    "        last_data_shift_list.append(tmp_ts[-i])\n",
    "        print(last_data_shift_list)\n",
    "        shift_ts = tmp_ts.shift(i)\n",
    "        shift_ts_list.append(shift_ts)\n",
    "        tmp_ts = tmp_ts - shift_ts\n",
    "    tmp_ts.dropna(inplace=True)\n",
    "    return tmp_ts\n",
    "\n",
    "# revive\n",
    "def predict_diff_recover(predict_value, d):\n",
    "    if isinstance(predict_value, float):\n",
    "        tmp_data = predict_value\n",
    "        for i in range(len(d)):\n",
    "            tmp_data = tmp_data + last_data_shift_list[-i-1]\n",
    "    elif isinstance(predict_value, np.ndarray):\n",
    "        tmp_data = predict_value[0]\n",
    "        for i in range(len(d)):\n",
    "            tmp_data = tmp_data + last_data_shift_list[-i-1]\n",
    "    else:\n",
    "        tmp_data = predict_value\n",
    "        for i in range(len(d)):\n",
    "            try:\n",
    "                tmp_data = tmp_data.add(shift_ts_list[-i-1])\n",
    "            except:\n",
    "                raise ValueError('What you input is not pd.Series type!')\n",
    "        tmp_data.dropna(inplace=True)\n",
    "    return tmp_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## draw image to show the relations between them\n",
    "index, shop = load_data_as_npdict()\n",
    "for i in [3,4,5,8,12,15,21,22]:\n",
    "    s = shop[i][-30:]\n",
    "    p = np.zeros(s.shape)\n",
    "    for ii in range(s.shape[0]):\n",
    "        if ii == 0:\n",
    "            p[ii] = p[ii] + s[ii]\n",
    "        else:\n",
    "            p[ii] = p[ii - 1] + s[ii]\n",
    "    plt.plot(s,color='red')\n",
    "    plt.show()\n",
    "    plt.plot(p,color='blue')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# show the distribute and tend\n",
    "index, shop = load_data_as_npdict()\n",
    "for i in [3,4,5,8,12,15,21,22]:\n",
    "    s = shop[i]\n",
    "    p = np.zeros(s.shape)\n",
    "    for ii in range(7,s.shape[0]):\n",
    "        if s[ii-7] < s[ii]:\n",
    "            p[ii] = 1\n",
    "        else:\n",
    "            p[ii] = 0\n",
    "\n",
    "    plt.plot(s[-90:],color='red')\n",
    "    plt.show()\n",
    "    plt.scatter(range(30),p[-30:],color='blue')\n",
    "    plt.axis([0,45,-2,2])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## define the difference and recover function\n",
    "def one_step_diff(series ,d = 0):\n",
    "    if d == 0:\n",
    "        return series\n",
    "    size = series.shape[0]\n",
    "    print(size)\n",
    "    result = series[:]\n",
    "    for i in range(size - 1, d - 1, -1):\n",
    "        result[i] = result[i] - result[i - d]\n",
    "\n",
    "    return result\n",
    "\n",
    "\n",
    "def one_step_recover(series, d=0):\n",
    "    if d == 0:\n",
    "        return series\n",
    "    size = series.shape[0]\n",
    "    result = series[:]\n",
    "    for i in range(d, size):\n",
    "        result[i] = result[i] + result[i - d]\n",
    "\n",
    "    return result\n",
    "\n",
    "## simple test\n",
    "pd_index = pd.date_range('2016-10-1', periods=30)\n",
    "df = pd.Series(np.append(shop[1][-15:], np.zeros(15)),index = pd_index,dtype='f')\n",
    "print(df)\n",
    "result = diff_ts(df,[5])\n",
    "result2 = one_step_diff(df, 5)\n",
    "print(result)\n",
    "print(result2)\n",
    "print(predict_diff_recover(result,[5]))\n",
    "print(one_step_recover(result2, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "pd_index = pd.date_range('2016-10-11', periods=35)\n",
    "print(pd_index)\n",
    "\n",
    "# output the image to analysis the specific shop with arima\n",
    "\n",
    "for i in [1,2,3,4,700]:\n",
    "    train = shop[i][-21:]\n",
    "    \n",
    "    \n",
    "    train_mean = np.mean(train)\n",
    "    for it in range(train.shape[0]):\n",
    "        if train[it] > train_mean * 3 or train[it] < train_mean * 0.1:\n",
    "            train[it] = train_mean\n",
    "    \n",
    "    train = np.append(shop[i][-21:],np.zeros(14))\n",
    "    train[:-14] = np.log(train[:-14])\n",
    "    df = pd.Series(train,index = pd_index, dtype='f')\n",
    "#     print(df)\n",
    "    d = 0\n",
    "    df_diff = df\n",
    "    while test_stationarity(df_diff) != True:\n",
    "        d += 1\n",
    "        df_diff = one_step_diff(df,d)\n",
    "    \n",
    "    dd = df_diff[d:-14]\n",
    "#     print(dd)\n",
    "#     df = pd.Series(,index = pd_index,dtype='f')\n",
    "#     df_log = np.log(df)\n",
    "\n",
    "    P = [0,1,2,3,4,5,6,7,8,9]\n",
    "    Q = [0,1,2,3,4,5,6,7,8,9]\n",
    "    p_b = 0\n",
    "    q_b = 0\n",
    "    model_b = None\n",
    "    bic_b = 999999999\n",
    "    for p in P:\n",
    "        for q in Q:\n",
    "            try:\n",
    "                model = ARMA(dd, order=(p,q))\n",
    "                result_arma = model.fit(disp=-1, method='css')\n",
    "                if result_arma.bic < bic_b:\n",
    "                    p_b = p\n",
    "                    q_b = q\n",
    "                    model_b = model\n",
    "                    bic_b = result_arma.bic\n",
    "            except:\n",
    "                continue\n",
    "        \n",
    "    print(d,p_b,q_b,bic_b)\n",
    "    model = model_b\n",
    "    result_arma = model.fit(disp=-1, method='css')\n",
    "    predict_ts = result_arma.predict()\n",
    "    df_diff[-14:] = result_arma.predict('2016-11-1', '2016-11-14', dynamic=True)\n",
    "    #print(predict_ts)\n",
    "    #print(df_diff)\n",
    "    \n",
    "    df_diff = one_step_recover(df_diff,d)\n",
    "    \n",
    "    recover = np.exp(df_diff)\n",
    "    predict_ts.plot(color='red')\n",
    "    recover[:-14].plot(color='blue')\n",
    "    recover[-14:].plot(color='red')\n",
    "    plt.show()\n",
    "    #print(recover)\n",
    "#     re_result = predict_diff_recover(predict_ts, d=[7])\n",
    "#     result = np.exp(re_result)\n",
    "\n",
    "\n",
    "#     df.plot(color='blue')\n",
    "#     result.plot(color='red')\n",
    "# #     rr = result_arma.predict('2016-11-1', '2016-11-14', dynamic=True)\n",
    "# #     rr.plot(color='green')\n",
    "#     plt.show()\n",
    "#     f = result_arma.predict('2016-11-1', '2016-11-14', dynamic=True)\n",
    "#     f = result_arma.predict()\n",
    "#     print(f)\n",
    "#     print(result_arma.forecast(10))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
